# _*_coding     : UTF_8_*_
# Author        :Jie Shen
# CreatTime     :2021/12/27 10:14

import time

import pandas as pd
from selenium import webdriver
import os
from pathlib import Path
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import pickle


def save_obj(obj, name):
    if not os.path.exists("obj"):
        os.mkdir("obj")
    with open('obj/' + name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)


def load_obj(name):
    with open('obj/' + name + '.pkl', 'rb') as f:
        return pickle.load(f)


def sleep(t=0.3):
    time.sleep(t)


class qcc:
    driver = None
    province_city_district = {}

    def __init__(self, driver_path):
        self.driver_path = driver_path
        self.province_city_district = load_obj("province_city_district")

    def pcd_to_csv(self, filename):
        """
        将self.province_city_district，转换成csv文件。
        该文件只带有省、市、区信息，并不带有高新技术企业数量。
        :return:
        """
        pcd_ = []
        for p in self.province_city_district.keys():
            if p == "offset":
                continue
            if "no" in p:
                continue

            for c in self.province_city_district[p].keys():
                List = []
                if c == "offset":
                    continue
                if "no" in c:
                    List.append(p)
                    List.append(None)
                    List.append(None)
                    List.append(None)
                    pcd_.append(pd.Series(List))
                    continue
                for d in self.province_city_district[p][c].keys():
                    List = []
                    if d == "offset":
                        continue
                    if "no" in d:
                        List.append(p)
                        List.append(c)
                        List.append(None)
                        List.append(None)

                        pcd_.append(pd.Series(List))
                        continue
                    List.append(p)
                    List.append(c)
                    List.append(d)
                    List.append(None)
                    pcd_.append(pd.Series(List))

        df = pd.DataFrame(pcd_)
        df.columns = ["省", "市", "区", "高新企业数"]
        df.to_csv(filename, encoding="gbk", index=False)

    def test(self):
        self.open_qcc()
        self.search_enterprise_num("广东省", "深圳市", "光明区")

    def read_qcc_csv(self, filename):
        """
        爬取高新企业数，填充到csv文件中
        :param filename:
        :return:
        """
        self.open_qcc()
        csv_ = pd.read_csv(filename, encoding="gbk")
        size = csv_.shape[0]
        for i in range(size):
            p = csv_.iloc[i, 0]
            c = csv_.iloc[i, 1]
            d = csv_.iloc[i, 2]
            num = csv_.iloc[i, 3]
            if not pd.isnull(num):
                continue
            print(p, c, d)
            if pd.isnull(c):
                c = None
            if pd.isnull(d):
                d = None
            num = self.search_enterprise_num(p, c, d)
            print(num)
            csv_.iloc[i, 3] = num
            csv_.to_csv(filename, index=False, encoding="gbk")

    def check_high_tech_enterprise(self):
        """

        :return:
        """
        js = "window.scrollTo(0, 100000)"
        self.driver.execute_script(js)

        Start = "//span[text()=\" 高新企业 \"]"
        element_to_hover_over = self.driver.find_element_by_xpath(Start)
        ActionChains(self.driver).move_to_element(element_to_hover_over).perform()  # 找到元素

        time.sleep(0.5)
        xpath_str = "//a[@title=\"高新技术企业\"]"
        self.driver.find_element_by_xpath(xpath_str).click();
        sleep()

        # 回到页面顶部
        js = "window.scrollTo(0, 0)"
        self.driver.execute_script(js)

    def open_qcc(self):
        self.driver = webdriver.Chrome(self.driver_path)
        self.driver.maximize_window()
        self.driver.get("https://www.qcc.com")
        sleep()
        win_str = "https://www.qcc.com/web/search/advance"
        self.driver.get(win_str)
        # 勾选高新技术企业
        self.check_high_tech_enterprise()
        # 悬停在 省份地区 ——> 全部地区
        xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/" \
                    "div[2]/div[4]/div[4]/div[2]/div/div/span/a/span"
        self.hover_by_xpath(xpath_str)

        # 悬停在第一个省
        xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/" \
                    "div[4]/div[4]/div[2]/div/div/div/div/div[1]/div/ul/li[1]"
        self.hover_by_xpath(xpath_str)

    def search_enterprise_num(self, province, city=None, district=None):
        # 检查当前选中状态
        div_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[2]/div/div[2]"
        div_ = self.driver.find_element_by_xpath(div_xpath)
        a_ = div_.find_elements_by_xpath("a")

        # 若a_为2.清除原先选中的省、市、区
        if len(a_) == 2:
            xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[2]/div/" \
                        "div[2]/a[2]/span"
            self.driver.find_element_by_xpath(xpath_str).click()
            sleep()

        if len(a_) > 2:
            print("暂不支持")
            return

        # 先悬停在 省份地区 ——> 全部地区
        xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/" \
                    "div[2]/div[4]/div[4]/div[2]/div/div/span/a/span"
        # js = "//div[@class=\"slimScrollDiv\"]/ul[@class=\"drop-col\"]"
        self.hover_by_xpath(xpath_str)

        # 悬停在第一个省
        xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/" \
                    "div[4]/div[4]/div[2]/div/div/div/div/div[1]/div/ul/li[1]"
        self.hover_by_xpath(xpath_str)

        p = self.province_city_district.get(province)
        p_offset = p.get("offset")
        self.province_scroll(p_offset)
        self.hover_province_city_district(province)

        if city is None:
            ok = self.click_province_city_district(province)
            if ok is True:
                return self.get_num()
            else:
                return None

        # 选中市
        c_offset = self.province_city_district[province][city].get("offset")
        self.hover_top_city()
        self.city_scroll(c_offset)
        self.hover_city(city)

        if district is None:
            ok = self.click_province_city_district(city)
            if ok is True:
                return self.get_num()
            else:
                return None
        # 选中区
        self.hover_top_district()
        # d_offset = self.province_city_district[province][city][district].get("offset")
        # self.district_scroll(d_offset)
        # 自动下滑直到该区出现

        found = self.find_district_text(district)
        if found is False:
            return None

        ok = self.click_province_city_district(district)
        if ok is True:
            return self.get_num()
        else:
            return None

    def get_num(self):
        """
        获取企业数量
        :return:
        """
        num_xpath = "/html/body/div[1]/div[2]/div[4]/div/span/span"
        return self.driver.find_element_by_xpath(num_xpath).text

    def hover_province_city_district(self, pcd):
        # 鼠标悬停在某个省，展开该省的市
        try:
            xpath_str = "//span[text()=\"{t}\"]".format(t=pcd)
            self.hover_by_xpath(xpath_str)
            return True
        except Exception as e:
            print(e.args)
            return False

    def hover_city(self, city):
        # 鼠标悬停在某个市
        try:
            xpath_str = "//span[text()=\"{t}\"]".format(t=city)
            self.hover_by_xpath(xpath_str)
            return True
        except Exception as e:
            print(e.args)
            return False

    def click_province_city_district(self, pcd):
        xpath_str = "//span[text()=\"{t}\"]".format(t=pcd)
        self.driver.find_element_by_xpath(xpath_str).click()
        sleep(0.5)
        return True

        # try:
        #     xpath_str = "//span[text()=\"{t}\"]".format(t=pcd)
        #     self.driver.find_element_by_xpath(xpath_str).click()
        #     sleep(0.5)
        #     return True
        # except Exception as e:
        #     print(e.args)
        #     return False

    def hover_top_city(self):
        try:
            city_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/" \
                         "div[4]/div[2]/div/div/div/div/div[2]/div/ul/li[1]"
            self.hover_by_xpath(city_xpath)
        except:
            city_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/" \
                         "div[4]/div[2]/div/div/div/div/div[2]/ul/li[1]"
            self.hover_by_xpath(city_xpath)
        sleep()

    def hover_by_xpath(self, xpath_str):
        element_to_hover_over = self.driver.find_element_by_xpath(xpath_str)
        ActionChains(self.driver).move_to_element(element_to_hover_over).perform()
        sleep()

    def hover_top_district(self):
        try:
            d_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/" \
                      "div[4]/div[2]/div/div/div/div/div[3]/div/ul/li[1]"
            self.hover_by_xpath(d_xpath)
        except Exception as e:
            d_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/" \
                      "div[4]/div[2]/div/div/div/div/div[3]/ul/li[1]"
            self.hover_by_xpath(d_xpath)

    def province_scroll(self, offset):
        """
        把省窗口，滚动条下滑到offset
        :param offset:
        :return:
        """
        js = 'document.getElementsByClassName("drop-col")[3].scrollTop={offset}'.format(
            offset=offset)
        self.driver.execute_script(js)
        sleep()

    def city_scroll(self, offset):
        js = 'document.getElementsByClassName("drop-col")[4].scrollTop={offset}'.format(
            offset=offset)
        self.driver.execute_script(js)
        sleep()

    def district_scroll(self, offset):
        js = 'document.getElementsByClassName("drop-col")[5].scrollTop={offset}'.format(
            offset=offset)
        self.driver.execute_script(js)
        sleep()

    def fill_province_city_district(self):
        """
        【注意】：
            1. 非必要无需调用此函数，因为会提供已经爬取完毕的文件 province_city_district.pkl。
                * 若丢失了 province_city_district.pkl，
                备用下载地址 https://gitee.com/jieshenai/spider/blob/master/selenium/example/obj/province_city_district.pkl
            2. 若必须调用，由于程序执行时间很长，确保电脑不会在中途进入睡眠、关机状态，造成程序中断。

        功能描述：
            爬取 www.qcc.com中的，省、市、区的信息，和对应在子窗口的offset;
            可根据对应的子窗口下滑offset，轻松定位到对应的省、市、区的信息。

        province_city_district.pkl的使用:
                def __init__(self, driver_path):
                    ...
                    # 从本地读取文件
                    self.province_city_district = load_obj("province_city_district")

        :return:
        """
        self.open_qcc()

        self.fill_province()

        self.fill_city()

        self.fill_district()
        # 先将self.province_city_district对象保存到本地
        save_obj(self.province_city_district, "province_city_district")
        print(self.province_city_district)

    def fill_district(self):

        # 悬停在第一个省
        xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/" \
                    "div[4]/div[4]/div[2]/div/div/div/div/div[1]/div/ul/li[1]"
        element_to_hover_over = self.driver.find_element_by_xpath(xpath_str)
        ActionChains(self.driver).move_to_element(element_to_hover_over).perform()
        sleep()
        p_offset = 0
        # 遍历所有的省
        for province in self.province_city_district.keys():
            if p_offset > 0:
                # 滑动滚动条找到对应的省
                js = 'document.getElementsByClassName("drop-col")[3].scrollTop={p_offset}'.format(
                    p_offset=p_offset)
                self.driver.execute_script(js)
            p_offset += 33

            # 悬停在该省
            xpath_str = "//span[text()=\"{t}\"]".format(t=province)
            element_to_hover_over = self.driver.find_element_by_xpath(xpath_str)
            ActionChains(self.driver).move_to_element(element_to_hover_over).perform()
            sleep()

            # 若该省没有市，跳过
            if "no_city" in self.province_city_district[province]:
                continue

            # 悬停在第一个市
            # city_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/" \
            #              "div[2]/div[4]/div[4]/div[2]/div/div/div/div/div[2]/div/ul/li[1]"
            # element_to_hover_over = self.driver.find_element_by_xpath(city_xpath)
            # ActionChains(self.driver).move_to_element(element_to_hover_over).perform()
            # sleep()
            c_offset = 0

            for city in self.province_city_district[province].keys():

                print(province, city, ">")
                if city == "offset":
                    continue
                # Offset = self.province_city_district[province][city].get("offset")
                # 下滑市的滚动条

                if c_offset > 0:
                    js = 'document.getElementsByClassName("drop-col")[4].scrollTop={c_offset}'.format(
                        c_offset=c_offset)
                    self.driver.execute_script(js)
                    sleep()
                c_offset += 33
                # 悬停在某个具体的市，使得页面展示该市的区
                xpath_str = "//span[text()=\"{city}\"]".format(city=city)
                element_to_hover_over = self.driver.find_element_by_xpath(xpath_str)
                ActionChains(self.driver).move_to_element(element_to_hover_over).perform()
                sleep()
                try:
                    # 试探是否该市存在区；若该市没有区，跳过该市；
                    xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/" \
                                "div[4]/div[4]/div[2]/div/div/div/div/div[3]"

                    self.driver.find_element_by_xpath(xpath_str)

                except Exception as e:
                    print(e)
                    self.province_city_district[province][city]["no_district"] = "no district info"
                    print(city, "no district")
                    continue
                d_offset = 0
                go = True
                while go:
                    if d_offset > 0:
                        js = 'document.getElementsByClassName("drop-col")[5].scrollTop={d_offset}'.format(
                            d_offset=d_offset)
                        self.driver.execute_script(js)
                    go = self.fill_district_text(province, city, d_offset)
                    sleep()
                    d_offset += 33

    def find_district_text(self, district):
        offset = 0

        while True:
            self.district_scroll(offset)
            offset += 200
            if offset > 8000:
                print()
                print("严重错误,no ", district)
                return False

            sleep()
            try:
                district_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/" \
                                 "div[4]/div[2]/div/div/div/div/div[3]/div/ul"
                _xpath = self.driver.find_element_by_xpath(district_xpath)
            except Exception as e:
                district_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/div[4]/" \
                                 "div[2]/div/div/div/div/div[3]/ul"
                _xpath = self.driver.find_element_by_xpath(district_xpath)

            _xpath = self.driver.find_element_by_xpath(district_xpath)
            selector = _xpath.find_elements_by_css_selector("li")

            for li in selector:
                text = li.find_element_by_xpath("./a/label/span").text.strip()
                if len(text) == 0:
                    continue
                # 找到该区
                if text == district:
                    self.hover_province_city_district(district)
                    return True

    def fill_district_text(self, province, city, offset):

        try:
            district_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/" \
                             "div[4]/div[2]/div/div/div/div/div[3]/div/ul"
            _xpath = self.driver.find_element_by_xpath(district_xpath)
        except Exception as e:
            print(province, city, "直接跟ul")
            district_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/div[4]/" \
                             "div[2]/div/div/div/div/div[3]/ul"
            _xpath = self.driver.find_element_by_xpath(district_xpath)

        _xpath = self.driver.find_element_by_xpath(district_xpath)
        selector = _xpath.find_elements_by_css_selector("li")
        fill = False
        for li in selector:
            text = li.find_element_by_xpath("./a/label/span").text.strip()
            if offset > 0:
                text = li.find_element_by_xpath("./a/label/span").text.strip()
            print(text, end=",")
            if len(text) == 0:
                continue
            if text not in self.province_city_district[province][city]:
                fill = True
                self.province_city_district[province][city][text] = {}
                self.province_city_district[province][city][text]["offset"] = offset
        print()
        return fill

    def fill_city(self):
        """
        填充所有省的区
        :return:
        """

        keys = self.province_city_district.keys()
        p_offset = 0
        for province in keys:

            # 滑动省滚动条
            js = 'document.getElementsByClassName("drop-col")[3].scrollTop={offset}'.format(
                offset=p_offset)
            self.driver.execute_script(js)
            p_offset += 33
            sleep()

            # 鼠标悬停在某个省，展开该省的市
            xpath_str = "//span[text()=\"{t}\"]".format(t=province)
            element_to_hover_over = self.driver.find_element_by_xpath(xpath_str)
            ActionChains(self.driver).move_to_element(element_to_hover_over).perform()
            sleep()

            try:
                # 悬停在第一个市
                city_xpath = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/" \
                             "div[2]/div[4]/div[4]/div[2]/div/div/div/div/div[2]/div/ul/li[1]"
                element_to_hover_over = self.driver.find_element_by_xpath(city_xpath)
                ActionChains(self.driver).move_to_element(element_to_hover_over).perform()
                sleep()
            except:
                # 有的省，没有市的信息。比如：香港、澳门、台湾、总局；
                self.province_city_district[province]["no_city"] = "no city info"
                continue

            # 滑动滚动条提取文本
            Offset = 0
            go = True
            while go:
                js = f'document.getElementsByClassName("drop-col")[4].scrollTop={Offset}'.format(
                    Offset=Offset)
                self.driver.execute_script(js)
                go = self.fill_city_text(province, Offset)
                sleep()
                Offset += 33

    def fill_city_text(self, province, offset):
        print(province)
        xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/" \
                    "div[4]/div[4]/div[2]/div/div/div/div/div[2]/div/ul"
        _xpath = self.driver.find_element_by_xpath(xpath_str)
        selector = _xpath.find_elements_by_css_selector("li")
        fill = False
        for li in selector:
            text = li.find_element_by_xpath("./a/label/span").text.strip()
            print(text, end=",")
            if len(text) == 0:
                continue
            if text not in self.province_city_district[province]:
                fill = True
                self.province_city_district[province][text] = {}
                self.province_city_district[province][text]["offset"] = offset
        print()
        return fill

    def fill_province(self):
        """
        爬取网页的省市区的信息
        鼠标要先悬停在 省份地区 -> 全部地区 ，不然爬取不到数据
        :return:
        """
        # 获取省的信息
        Offset = 0
        go = True
        while go:
            js = f'document.getElementsByClassName("drop-col")[3].scrollTop={Offset}'.format(
                Offset=Offset)
            self.driver.execute_script(js)
            go = self.fill_province_text(Offset)
            sleep()
            Offset += 33

    def fill_province_text(self, offset):

        """
        * 只要有新的省份就返回True,以便滚动栏继续下滑获取新的省份
        * 不再有新的省份，返回False，滚动栏停止下滑(所有省份获取完毕)
        :return:
        """
        fill = False
        xpath_str = "/html/body/div[1]/div[2]/div[1]/div/div[1]/div/div[2]/div[4]/div[4]/div[2]/div/div/div/" \
                    "div/div[1]/div/ul"
        xpath = self.driver.find_element_by_xpath(xpath_str)
        selector = xpath.find_elements_by_css_selector("li")
        for li in selector:
            text = li.find_element_by_xpath("./a/label/span").text.strip()
            if len(text) == 0:
                continue
            if text not in self.province_city_district:
                fill = True
                self.province_city_district[text] = {}
                self.province_city_district[text]["offset"] = offset
        return fill

    def __del__(self):
        # self.driver.quit()，无法关闭chromedriver，会导致内存泄露。即使程序结束，chromedriver也会一直占用内存；
        # bug说明: https://github.com/SeleniumHQ/selenium/issues/8571

        # pass
        sleep(2)
        try:
            self.driver.quit()
        except:
            # dos命令杀死chromedriver进程即可
            os.system("taskkill /F /im chromedriver.exe")


if __name__ == '__main__':
    # driver_path = r"D:\develop\chromedriver_win32\chromedriver.exe"
    driver_path = os.path.join("../../", "driver", "chromedriver.exe")
    # url = "https://www.qcc.com/web/search"
    filename = "out.csv"
    # q = qcc(r"D:\1_git\spider\selenium\driver\chromedriver.exe")
    q = qcc(driver_path)
    # q.fill_province_city_district()
    # q.pcd_to_csv("out.csv")
    q.read_qcc_csv(filename)
